{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pre-training VGG16 for Distillation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch \n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import Dataset, random_split, DataLoader\n",
    "\n",
    "from torchvision import datasets, transforms, models\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data import Dataset, random_split, DataLoader\n",
    "from torchvision.utils import save_image\n",
    "\n",
    "from torchsummary import summary\n",
    "\n",
    "import spacy\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import os\n",
    "import time\n",
    "import math\n",
    "from PIL import Image\n",
    "import glob\n",
    "from IPython.display import display"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.manual_seed(0)\n",
    "np.random.seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 32\n",
    "LR = 5e-4\n",
    "NUM_EPOCHES = 25"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "transform = transforms.Compose([\n",
    "    transforms.RandomHorizontalFlip(),\n",
    "    #transforms.RandomVerticalFlip(),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.5,), (0.5,))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "trainset = datasets.CIFAR10('../data/CIFAR10/', download=True, train=True, transform=transform)\n",
    "trainloader = torch.utils.data.DataLoader(trainset, batch_size=BATCH_SIZE, shuffle=True)\n",
    "\n",
    "testset = datasets.CIFAR10('../data/CIFAR10/', download=True, train=False, transform=transform)\n",
    "testloader = torch.utils.data.DataLoader(testset, batch_size=BATCH_SIZE, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformer_package.models.transformer import VGG16_classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: \"https://download.pytorch.org/models/vgg16-397923af.pth\" to C:\\Users\\udbha/.cache\\torch\\hub\\checkpoints\\vgg16-397923af.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bebb657b462a4d80bf6fe98329eeb1a1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0.00/528M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "VGG16_classifier(\n",
       "  (vgg16): VGG(\n",
       "    (features): Sequential(\n",
       "      (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (1): ReLU(inplace=True)\n",
       "      (2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (3): ReLU(inplace=True)\n",
       "      (4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "      (5): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (6): ReLU(inplace=True)\n",
       "      (7): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (8): ReLU(inplace=True)\n",
       "      (9): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "      (10): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (11): ReLU(inplace=True)\n",
       "      (12): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (13): ReLU(inplace=True)\n",
       "      (14): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (15): ReLU(inplace=True)\n",
       "      (16): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "      (17): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (18): ReLU(inplace=True)\n",
       "      (19): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (20): ReLU(inplace=True)\n",
       "      (21): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (22): ReLU(inplace=True)\n",
       "      (23): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "      (24): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (25): ReLU(inplace=True)\n",
       "      (26): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (27): ReLU(inplace=True)\n",
       "      (28): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (29): ReLU(inplace=True)\n",
       "      (30): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    )\n",
       "    (avgpool): AdaptiveAvgPool2d(output_size=(7, 7))\n",
       "    (classifier): Sequential(\n",
       "      (0): Linear(in_features=25088, out_features=2048, bias=True)\n",
       "      (1): ReLU()\n",
       "      (2): Dropout(p=0.3, inplace=False)\n",
       "      (3): Linear(in_features=2048, out_features=1024, bias=True)\n",
       "      (4): ReLU()\n",
       "      (5): Dropout(p=0.3, inplace=False)\n",
       "      (6): Linear(in_features=1024, out_features=512, bias=True)\n",
       "      (7): ReLU()\n",
       "      (8): Dropout(p=0.3, inplace=False)\n",
       "      (9): Linear(in_features=512, out_features=10, bias=True)\n",
       "    )\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classes = 10\n",
    "hidden_size = 512\n",
    "dropout = 0.3\n",
    "\n",
    "model = VGG16_classifier(classes, hidden_size, preprocess_flag=False, dropout=dropout).to(device)\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input Image Dimensions: torch.Size([32, 3, 32, 32])\n",
      "Label Dimensions: torch.Size([32])\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Output Dimensions: torch.Size([32, 10])\n"
     ]
    }
   ],
   "source": [
    "for img, label in trainloader:\n",
    "    img = img.to(device)\n",
    "    label = label.to(device)\n",
    "    \n",
    "    print(\"Input Image Dimensions: {}\".format(img.size()))\n",
    "    print(\"Label Dimensions: {}\".format(label.size()))\n",
    "    print(\"-\"*100)\n",
    "    \n",
    "    out = model(img)\n",
    "    \n",
    "    print(\"Output Dimensions: {}\".format(out.size()))\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(params=model.parameters(), lr=LR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------------\n",
      "Epoch: 1 Train mean loss: 2154.38713694\n",
      "       Train Accuracy%:  45.754 == 22877 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 2 Train mean loss: 1405.58220522\n",
      "       Train Accuracy%:  69.2 == 34600 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 3 Train mean loss: 1092.48749158\n",
      "       Train Accuracy%:  78.818 == 39409 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 4 Train mean loss: 962.41183120\n",
      "       Train Accuracy%:  81.22 == 40610 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 5 Train mean loss: 834.71479223\n",
      "       Train Accuracy%:  84.07 == 42035 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 6 Train mean loss: 789.83483182\n",
      "       Train Accuracy%:  85.104 == 42552 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 7 Train mean loss: 719.01736114\n",
      "       Train Accuracy%:  86.444 == 43222 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 8 Train mean loss: 729.94068751\n",
      "       Train Accuracy%:  86.226 == 43113 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 9 Train mean loss: 635.71813869\n",
      "       Train Accuracy%:  87.956 == 43978 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 10 Train mean loss: 638.17087806\n",
      "       Train Accuracy%:  88.13 == 44065 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 11 Train mean loss: 574.59144469\n",
      "       Train Accuracy%:  89.37 == 44685 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 12 Train mean loss: 672.17665105\n",
      "       Train Accuracy%:  87.45 == 43725 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 13 Train mean loss: 578.20310671\n",
      "       Train Accuracy%:  89.342 == 44671 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 14 Train mean loss: 601.84968356\n",
      "       Train Accuracy%:  88.82 == 44410 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 15 Train mean loss: 517.88309733\n",
      "       Train Accuracy%:  90.582 == 45291 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 16 Train mean loss: 479.18201526\n",
      "       Train Accuracy%:  90.992 == 45496 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 17 Train mean loss: 519.36501931\n",
      "       Train Accuracy%:  90.382 == 45191 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 18 Train mean loss: 449.15972432\n",
      "       Train Accuracy%:  91.748 == 45874 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 19 Train mean loss: 498.29070443\n",
      "       Train Accuracy%:  91.034 == 45517 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 20 Train mean loss: 452.45193216\n",
      "       Train Accuracy%:  91.908 == 45954 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 21 Train mean loss: 449.59901455\n",
      "       Train Accuracy%:  92.108 == 46054 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 22 Train mean loss: 455.93260922\n",
      "       Train Accuracy%:  91.628 == 45814 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 23 Train mean loss: 459.63933291\n",
      "       Train Accuracy%:  91.79 == 45895 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 24 Train mean loss: 608.39501254\n",
      "       Train Accuracy%:  90.672 == 45336 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 25 Train mean loss: 435.79778930\n",
      "       Train Accuracy%:  92.314 == 46157 / 50000\n",
      "-------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "loss_hist = {}\n",
    "loss_hist[\"train accuracy\"] = []\n",
    "loss_hist[\"train loss\"] = []\n",
    "\n",
    "for epoch in range(1, NUM_EPOCHES+1):\n",
    "    model.train()\n",
    "    \n",
    "    epoch_train_loss = 0\n",
    "        \n",
    "    y_true_train = []\n",
    "    y_pred_train = []\n",
    "        \n",
    "    for batch_idx, (img, labels) in enumerate(trainloader):\n",
    "        img = img.to(device)\n",
    "        labels = labels.to(device)\n",
    "        \n",
    "        preds = model(img)\n",
    "        \n",
    "        loss = criterion(preds, labels)\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        y_pred_train.extend(preds.detach().argmax(dim=-1).tolist())\n",
    "        y_true_train.extend(labels.detach().tolist())\n",
    "            \n",
    "        epoch_train_loss += loss.item()\n",
    "    \n",
    "    loss_hist[\"train loss\"].append(epoch_train_loss)\n",
    "    \n",
    "    total_correct = len([True for x, y in zip(y_pred_train, y_true_train) if x==y])\n",
    "    total = len(y_pred_train)\n",
    "    accuracy = total_correct * 100 / total\n",
    "    \n",
    "    loss_hist[\"train accuracy\"].append(accuracy)\n",
    "    \n",
    "    print(\"-------------------------------------------------\")\n",
    "    print(\"Epoch: {} Train mean loss: {:.8f}\".format(epoch, epoch_train_loss))\n",
    "    print(\"       Train Accuracy%: \", accuracy, \"==\", total_correct, \"/\", total)\n",
    "    print(\"-------------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAhUklEQVR4nO3deXRc9X338fdX+77Zkmwsyys22IANFoZAQkNYk4aYkISQEkITUtIenqakTQr06QlNW3JI2yxNn540DiSBBNISAgE3HGLjhCVhtQ3eF7zIkixZkrVLM9q/zx8zMsKRbcnWaKS5n9c5c2bmapbv9RzPZ+7v/hZzd0REJLiS4l2AiIjEl4JARCTgFAQiIgGnIBARCTgFgYhIwKXEu4DRmD59us+dOzfeZYiITCkbN2484u7FJ3vclAiCuXPnsmHDhniXISIypZjZwdE8Tk1DIiIBpyAQEQk4BYGISMApCEREAk5BICIScAoCEZGAUxCIiAScgkBEZBJq6Ojma2u20xbui/l7TYkBZSIiAP0Dg7SF+/7g0n7M/aLsdBaW5LCwJIcFxdnkZqTGu/RR6+rpZ/WL+/nBS/vp7R/k0gXTuXJJaUzfU0EgIqeku2+A1OQkkpNs3F+7LdTHC2838ttdDew63HH0i76zp/+Ez8tMTSY3I4WWUC99A+8sujUjL+NoMAy/TMtOw+zE9bs7Pf2DdPb009ndT2dPPx3d/eRlprBkZt5Jnz9a/QODPLahhm8/t4fGjh7++NyZfOWaxcydnj0ur38iCgIRGZO9DZ088NJ+nth0iPSUJFbMLeTCuUWsnFfEeWX5pKckj/k13Z3d9R38dlfky39jVQsDg05hVirnlxdy9sxcCjLTyM9MJT8zhfys1OjtoW2RS1pKpLW7f2CQquYQexs6ebuhk30Nnext7OSxDdWEegeOvm9BVioLi3Mon5ZF34DT2d139Iu+sydy6erpf1eoDLewJIePryjjo+fPojQv45T+Pd2d9TsbuP/ZXext6KRiTiHfv2UFF5QXntLrnQqbCktVVlRUuOYaEokfd2fDwRa+/8J+nttZT3pKEjdcMAsw3qhsZm9DJwBpKUksn13AyrlFXDiviBVzCslJH/n3Zrh3gJf3HeE3uxp4fncjh1rDACyZmccHzirh8rNKWD67YFyPONyd2rZu9jZ0Hr3sa+ikuiVERmoyOekpkUtGCrnpKWRHb+ekp5AbvR76+8GmEL/YWMOGgy0kGVy2qJiPXVDGVUtKyUgdXRhurm7l68/s5LUDzcyfns1dHzyLq5eUjttRhpltdPeKkz5OQSAixzMw6KzbcZjvv7ifN6taKcxK5Zb3zOUz75nD9Jz0o49r7urljcpm3jjQzBuVzWyrbWdg0EkyWHJGHivnTmPlvELmTc/htQNN/GZXA6/sa6Knf5CstGQuXTg98uW/uIQZ+af2yzpe9jd28sSmQ/xiUw11bd3kZaRw3bIz+PiKMpbPLhjxS72qKcS/rt3Nms21TMtO484rz+SmleWkJo9v/x0FgUiCc3caO3uoagpR2RSiqqmLg80hDjaFaAn1sqg0l2Vl+ZxbVsB5s/IpzE4b9Wt39w3wi001PPDSAQ4c6aK8KIvPv28eH19RRlbayVuUu3r62VTVwhsHmnm9spk3q1rp6R88+ve507K4PPrFf9H8olNqTppsBgadV/Y18fjGap7dfpjuvkEWFGfzsRVl3HB+GTPyM2gN9fIfv9nLw69Ukpxk/Nn75nP7ZfNjdjJbQSCSIBrau9ld38HBphBVzSEqj3RR1Ry5Pby9O8lgZn4mc6ZlUZCVyq66DvYf6Tr699lFmZw3q4Bzy/I5ryyfc2blk3fMF1BLVy8/efUgD71cSVNXL+eV5fOFyxZw7TkzTquJprd/kK2H2tjf2MmKOYXML8455deaCjq6+3hmax2Pb6zhjcpI09FF86axvbaNzp5+PrFiNl+6alHMj34mRRCY2V8BfwYY8AN3/46ZFQH/A8wFKoEb3b3lRK+jIJAgagv38Z3n9vDwKwcZGIz8P01LSaK8KIs5RVnMmZbNnGlZlE+L3C8rzDp6snRIe3cf2w61sbWmjS01bWw51Ep1c/jo3+dPz44GQwFVTV08tqGGcN8Aly8u5vbLFnDx/KJxa68OqsojXfxiUw2/2lLH3OnZ/O21izlrRt6EvHfcg8DMzgH+G1gJ9ALPAn9BJBia3f1+M7sbKHT3u070WgoCCZLBQefxjTV849ldNId6+dTKcj6y7AzmTMuiNDeDpNM8edrS1cvWQ21sqWllS00bWw+1UdfWTWqysWr5LG6/bD6LSnPHaW8knkYbBLHsPno28Kq7h6IFvQB8FFgFvD/6mIeA54ETBoFIULxV3cq9T29nc3UrK+YU8tBHVnLOrPxxfY/C7DQuW1TMZYveWcGwoaOblKQkisZwHkESRyyDYBtwn5lNA8LAh4ANQKm71wG4e52ZlYz0ZDO7HbgdoLy8PIZlisTfkc4e/uXZXTy2oYbi3HS+deMyPnr+rAlrlinJnVo9dWR8xSwI3H2nmX0DWAd0ApuBEw8LfPfzVwOrIdI0FJMiReKsf2CQh185yLef20O4d4DbL5vPX35g4ZSaEkGmvpiOLHb3B4EHAczs60ANUG9mM6NHAzOBhljWIMHS0tXLzzdW89+vV5Oflco/rTpn3JtWAHr6B/iv5/ezZkst5UVZLJ6Ry+LSXBbPyGVBcc4fnLQdycv7jvAPT29nT30n7ztzOvdet5SFJYndm0Ymp5gGgZmVuHuDmZUDNwDvAeYBtwL3R6+fimUNEgxvVbfyk1cOsmZLLb39g1w4t5DKphCr/vP3fP598/jSlYtGPdrzZN6obOaeJ7ayt6GTi+YVUdsa5qW3G49OQ5CSZMwvzmbxjDwWl+aweEYeZ83IZVZBJklJRm1rmPue2cmvttRRVpjJ929ZMa6jSUXGKtZzDf0ieo6gD7jD3VvM7H7gMTO7DagCPhHjGiRBhXsHWLOllp++epAtNW1kpyVzY0UZt1w8l8UzcmkL9fH1Z3by/Rf28+tth/n6DedyyYLpp/x+beE+/uXZXTzyWhWzCjL50Wcv5PLFkVNcvf2DHDjSxa7D7eyp72D34Q7erGphzebao8/PTkvmzNJcdh/uYNCdL125iC/80fxxCyiRU6UBZTLlHDjSxSOvHuTnG2toC/dxZkkOn3nPHK4/f9aIbesv7z3CPU9u5WBTiJsunM09Hzqb/MzRt8G7O89uO8y9T2/nSGcPn7t0Hl+6ahHZx5lDZ7iO7j721HceDYddh9uZmZ/J31y9iLLCrDHtt8hYxX0cwXhSEMjAoPObXQ385NWDvLinkZQk45pzZnDLxXO4aN7JBz2Fewf4zvo9PPDSAYqy0/inVUu59pyZJ33f2tYwX31qO8/trGfpGXncf8N5nFs2/uccRGJBQSCT1uCg09HTT8ewKX87uvvo6O6nfdjtd6772VXXTm1bN6V56fzJyjl8auVsSk5h2t9th9r428e3sKOunWuWlvKPq84ZcfrggUHnJ69U8q+/3s2AO3991SI+d+k8UsZ5UjCRWFIQyKTT3TfAg787wPee33fSBUZSkozcjBRyM1LJzUhhZn4GH19RxhVnl572DI19A4M88NIBvvPcHtJSkvi7D53NTRfOPnpUsbOunbuf2Mrm6lYuW1TMfdefw+wiNePI1KMgkEnD3fn19nrue2YH1c1hrjy7lIvnF73ri/6d6xTyMlJJT0mKeS+aA0e6uOeJLby6v5mL5xdx73VLWbO5ltUv7ic/M5WvXreEjyw7Q715ZMpSEMiksOtwO/+4Zgcv72tiUWkOX/3wUt575qn33Blv7s7/vFHNfc/spKM7cpTy8RVl/N8PnT2maZtFJqPJMNeQBFhLVy/ffm4PP331ILkZqfzjqqX8ycrySdfGbmbctLKcy88q4Qcv7ucDZ5VwycLJE1QiE0FBIOOqf2CQR16r4lvr9tDZ088tF8/hzisXTfpf16V5Gfz9h5fEuwyRuFAQyLj5/d4jfG1NZMqESxZM497rlrJ4hqYzFpnsFAQCvLM27esHWsjNSKEwK5XC7DTyM1MpzEqjICuVgqw08jJS/uDk6cGmLu771U7W7qhndpGmTBCZahQEAdfV08/jG2v44e8PcLApRHpK0rvWlj1WcpKRn5lKQVYkIHLSU3hlXxMpycZXrlnMbe+dpykTRKYYBUFA1bd38+OXK3n0tSrawn2cX17AXdeexTVLZ+DutHf30xLqpTXUS0tXH63hvsjtUC+toT5aQ320hHpp6Ohh1fIz+PI1i0ccmCUik5+CIGB21LbzwO/2s2ZzLQODzjVLZ/D5981nxZzCYY8yirLTtFqVSEAoCALA3Xl+TyMPvnSA3+09QlZaMjdfNIfPXTqP8mkaMSsSdAqCBNbdN8BTbx3igZcO8HZDJ6V56dx17Vn8ycpy8rO0ApaIRCgIEoy7s6mqlTWba1mzuZamrl7OnpnHt25cxofPO2NUK2eJSLAoCBKAu7Ojrp01m+tYs7mWQ61h0lKSuOKsEj598RwuWTBNXTlF5LgUBFPY/sZO1myu4+nNh9jX2EVykvG+M6fz11ct4uqlpVoAXURGRUEwxRxqDfO/m2tZs6WWbYfaMYOVc4v43Hvn8cFzZqqnj4iMmYJgivjfLbX8+PeVbDjYAsCysnz+/o/P5sPnncGMfPXfF5FTpyCYAr73/D6+8ewuFhRn8+WrF3HdsjOYMy073mWJSIJQEExi7s631u3hP36zl48sO4Nv3rjstFfnEhE5loJgknJ3/vlXO3nwdwf4ZMVsvn7DuSQnqeePiIw/BcEkNDDo/P0vt/Gz16v400vm8tUPLyFJISAiMaIgmGT6Bwb58s8388u3arnj8gV8+erFGgMgIjGlIJhEevoH+OLP3uTX2+v5yjWLuePyhfEuSUQCQEEwSYR7B/jzn27khT2N3HvdEj576bx4lyQiAaEgmAQ6e/q57cdv8HplM9/42Ll88sLyeJckIgGiIIiztlAfn/nR62w71MZ3PrmcVctnxbskEQkYBUEcHens4ZYHX2dfQyffu/kCrl46I94liUgAKQji5HBbNzc/8CqHWsM8cGsFly0qjndJIhJQCoI4qGoKcfODr9LS1cfDn7uIlfOK4l2SiASYgmCC/X7vEe54dBPu8NPPX8Ty2QXxLklEAk5BMEHcnQd/d4CvP7OTBcU5rP5MBfOma+I4EYm/mAaBmX0J+DzgwFbgs0AW8D/AXKASuNHdW2JZR7x19w1wzxNbefLNQ1yztJRv3ricnHRlsIhMDjGbytLMZgFfBCrc/RwgGbgJuBtY7+5nAuuj9xPWodYwH/+vl3nyzUP8zVWL+N7NKxQCIjKpxPobKQXINLM+IkcCtcA9wPujf38IeB64K8Z1xMWr+5u445FN9PQP8sBnKrhySWm8SxIR+QMxOyJw90PAvwFVQB3Q5u5rgVJ3r4s+pg4oiVUN8eLuPPRyJZ9+4DXys1L55R2XKgREZNKK2RGBmRUCq4B5QCvwczP79BiefztwO0B5+dSZcqG7b4CvPrWNxzbUcMVZJXz7puXkaRF5EZnEYtk0dCVwwN0bAczsCeASoN7MZrp7nZnNBBpGerK7rwZWA1RUVHgM6xw3h9u6+cJPN7K5upUvfmAhd165SOsIiMikF8sgqAIuNrMsIAxcAWwAuoBbgfuj10/FsIYJs6GymT//6SbCvf3816cv4NpzZsa7JBGRUYlZELj7a2b2OLAJ6AfeJPILPwd4zMxuIxIWn4hVDRPl0dequPfpbcwqyOTRP7uIRaW58S5JRGTUYtpryN3vBe49ZnMPkaODhLDxYDN/9+RW/mhRMd+96Xzys3Q+QESmFnVoP03PbD1MWnIS/3nzBRofICJTUsy6jwaBu7NuRz2XLpymEBCRKUtBcBp213dQ1RziqiVaR0BEpi4FwWlYt70eM7hyScKNiRORAFEQnIa1O+pZPruAktyMeJciInLKFASnqLY1zNZDbVytZiERmeIUBKfouZ31AFylOYREZIpTEJyidTvqmV+czcKSnHiXIiJyWhQEp6At3Mcr+5p0NCAiCUFBcAqe391A/6Dr/ICIJAQFwSlYu6Oe6TnpnK+F50UkASgIxqinf4AXdjdy1ZISTTEtIglBQTBGr+xrorOnX+cHRCRhKAjGaN2OerLSkrlkwfR4lyIiMi4UBGMwOBiZZO6PFhWTkZoc73JERMaFgmAMthxqo6Gjh6uXqllIRBKHgmAM1m4/THKScfliTTInIolDQTAG63bUc9G8Igqy0uJdiojIuFEQjNKBI1283dCp3kIiknAUBKO0bsdhQJPMiUjiURCM0trt9SyZmUdZYVa8SxERGVcKglE40tnDxqoW9RYSkYSkIBiF9TvrcVezkIgkJgXBKKzdXs+sgkyWzMyLdykiIuNOQXASXT39vLT3CFctKcVMk8yJSOJREJzES2830ts/qPMDIpKwFAQnsXZHPfmZqaycWxTvUkREYkJBcAL9A4P8ZlcDV5xVQkqy/qlEJDHp2+0E3qhsoTXUp95CIpLQFAQnsHbHYdJSkrhsUXG8SxERiRkFwXG4R9YeeO/C6WSnp8S7HBGRmFEQHMfOug5qWsJcrWYhEUlwCoLjWLejHjO44mwFgYgktlEFgZllm1lS9PYiM/uImaXGtrT4WrvjMBeUF1Kcmx7vUkREYmq0RwQvAhlmNgtYD3wW+PGJnmBmi83srWGXdjO708yKzGydmb0dvS48vV0Yf4daw2yvbVdvIREJhNEGgbl7CLgB+A93/yiw5ERPcPfd7r7c3ZcDK4AQ8CRwN7De3c8kEip3n2rxsbJue2TtAZ0fEJEgGHUQmNl7gJuBX0W3jaUrzRXAPnc/CKwCHopufwi4fgyvMyHW7axnQXE284tz4l2KiEjMjTYI7gTuAZ509+1mNh/47Rje5ybgZ9Hbpe5eBxC9HnEleDO73cw2mNmGxsbGMbzV6WkL9fHq/mauXjpjwt5TRCSeRvWr3t1fAF4AiJ40PuLuXxzNc80sDfgIkSAZNXdfDawGqKio8LE893T8dncDA4Ou8wMiEhij7TX0qJnlmVk2sAPYbWZfGeV7fBDY5O710fv1ZjYz+rozgYaxFh1La3ccpjg3neVlBfEuRURkQoy2aWiJu7cTac9/BigHbhnlcz/FO81CAE8Dt0Zv3wo8NcrXmRCv7Gvi/YuKSUrS2gMiEgyjDYLU6LiB64Gn3L0POGlzjZllAVcBTwzbfD9wlZm9Hf3b/WOqOIY6uvtoCfWxoEQniUUkOEbb8+f7QCWwGXjRzOYA7Sd7UrTL6bRjtjUR6UU06dS0hAEoK8yMcyUiIhNntCeLvwt8d9img2Z2eWxKip/q5hAAswuz4lyJiMjEGe3J4nwz+9ZQd04z+yaQHePaJtzQEcHsIgWBiATHaM8R/BDoAG6MXtqBH8WqqHipbgmRlZZMYVZCT6MkIvIuoz1HsMDdPzbs/tfM7K0Y1BNX1c1hZhdmYaYeQyISHKM9Igib2XuH7pjZpUA4NiXFT01LSCeKRSRwRntE8OfAw2aWH73fwjtjARKCu1PTEubi+dNO/mARkQQy2l5Dm4FlZpYXvd9uZncCW2JY24RqC/fR2dOvIwIRCZwxrVDm7u3REcYAfx2DeuKmunloDIF6DIlIsJzOUpUJdUa1uiU6hqBIRwQiEiynEwQTNiPoRKiJBoGOCEQkaE54jsDMOhj5C9+AhPrpXN0cJi8jhfxMjSEQkWA5YRC4e+5EFRJv1S0hjSgWkUA6naahhFLTElaPIREJJAUBQ2MIQppsTkQCSUEAHOnspbtvUE1DIhJICgLe6TqqpiERCSIFAcPWIdARgYgEkIKAd9YhmFWgIwIRCR4FAZHBZNOy08hOH+0cfCIiiUNBQGQwWZmahUQkoBQEaB0CEQm2wAfBwKBzqDWsMQQiEliBD4KGjm76BlyzjopIYAU+CLQOgYgEnYJgaAyBzhGISEAFPgiGxhCcoTEEIhJQgQ+C6pYQpXnpZKQmx7sUEZG4CHwQaNZREQm6wAdBdbPWIRCRYAt0EPQNDFLXFtZkcyISaIEOgsNt3Qy6pp8WkWALdBC803VURwQiElzBDoIWrUMgIhLTIDCzAjN73Mx2mdlOM3uPmRWZ2Tozezt6XRjLGk6kpiVMksGM/Ix4lSAiEnexPiL4d+BZdz8LWAbsBO4G1rv7mcD66P24qG4OMTM/k9TkQB8YiUjAxewb0MzygMuABwHcvdfdW4FVwEPRhz0EXB+rGk6mpiWsyeZEJPBi+VN4PtAI/MjM3jSzB8wsGyh19zqA6HXJSE82s9vNbIOZbWhsbIxJgdUtIU02JyKBF8sgSAEuAL7n7ucDXYyhGcjdV7t7hbtXFBcXj3tx3X0D1Lf3qMeQiAReLIOgBqhx99ei9x8nEgz1ZjYTIHrdEMMajqu2dWj6aTUNiUiwxSwI3P0wUG1mi6ObrgB2AE8Dt0a33Qo8FasaTqQ6Ouuouo6KSNClxPj1/xJ4xMzSgP3AZ4mEz2NmdhtQBXwixjWMqOboGAIdEYhIsMU0CNz9LaBihD9dEcv3HY3q5jCpyUZJrsYQiEiwBbYDfXVLiFkFmSQnWbxLERGJq8AGQWQMgc4PiIgENwiaQ+oxJCJCQIOgq6efpq5eDSYTESGgQXBIYwhERI4KZBAcXYdA5whERIIZBDVDg8nUNCQiEswgqG4OkZGaxPSctHiXIiISd8EMguiso2YaQyAiEsggqGkJ60SxiEhUIIOgujmk8wMiIlGBC4K2cB/t3f2abE5EJCpwQTA066gGk4mIRAQuCKqb1XVURGS4wAWB1iEQEXm3AAZBmJz0FPIzU+NdiojIpBC4IKiOzjqqMQQiIhGBC4LIGAKdHxARGRKoIHB3qltCOj8gIjJMoIKgJdRHqHdAPYZERIYJVBAMTT+t6SVERN4RrCBo0ToEIiLHClQQDK1DoCMCEZF3BCoIqptDFGSlkpuhMQQiIkOCFQQtYZ0oFhE5RqCCoKYlpGYhEZFjBCYIBgedmpawThSLiBwjMEFwpLOH3v5BZuuIQETkXQITBNVah0BEZETBCYKhdQg0vYSIyLsEJgiG1iGYVaAjAhGR4QITBNXNYabnpJOZlhzvUkREJpXABEFNq2YdFREZSWCCoLpZ6xCIiIwkpkFgZpVmttXM3jKzDdFtRWa2zszejl4XxrIGgIFBp7Y1rK6jIiIjmIgjgsvdfbm7V0Tv3w2sd/czgfXR+zF1uL2b/kHXYDIRkRHEo2loFfBQ9PZDwPWxfkOtQyAicnyxDgIH1prZRjO7Pbqt1N3rAKLXJSM90cxuN7MNZrahsbHxtIoYCgJNOCci8odSYvz6l7p7rZmVAOvMbNdon+juq4HVABUVFX46RdS0hDGDmQUZp/MyIiIJKaZHBO5eG71uAJ4EVgL1ZjYTIHrdEMsaIDK9xIy8DNJTNIZARORYMQsCM8s2s9yh28DVwDbgaeDW6MNuBZ6KVQ1DarQOgYjIccWyaagUeNLMht7nUXd/1szeAB4zs9uAKuATMawBgJrmEBfPnxbrtxERmZJiFgTuvh9YNsL2JuCKWL3vsXr7B6lr76ZMXUdFREaU8COL69rCuKvrqIjI8SR8EBydflrnCERERpTwQTA0/bQmnBMRGVnCB0F1S4jkJGNGnsYQiIiMJPGDoDnMGQUZpCQn/K6KiJyShP92rGkJ6fyAiMgJJHwQVLeE1WNIROQEEjoIuvsGaOzo0RGBiMgJJHQQ1LREuo6WqceQiMhxJXQQVLdo+mkRkZNJ6CAYOiLQymQiIseX2EHQHCItJYninPR4lyIiMmkldBDMm57NR5fPIinJ4l2KiMikFesVyuLqppXl3LSyPN5liIhMagl9RCAiIienIBARCTgFgYhIwCkIREQCTkEgIhJwCgIRkYBTEIiIBJyCQEQk4Mzd413DSZlZI3DwFJ8+HTgyjuVMNUHef+17cAV5/4fv+xx3Lz7ZE6ZEEJwOM9vg7hXxriNegrz/2vdg7jsEe/9PZd/VNCQiEnAKAhGRgAtCEKyOdwFxFuT9174HV5D3f8z7nvDnCERE5MSCcEQgIiInoCAQEQm4hA4CM7vWzHab2V4zuzve9UwkM6s0s61m9paZbYh3PbFmZj80swYz2zZsW5GZrTOzt6PXhfGsMVaOs+//YGaHop//W2b2oXjWGCtmNtvMfmtmO81su5n9VXR7UD774+3/mD7/hD1HYGbJwB7gKqAGeAP4lLvviGthE8TMKoEKdw/EoBozuwzoBB5293Oi2/4FaHb3+6M/BArd/a541hkLx9n3fwA63f3f4llbrJnZTGCmu28ys1xgI3A98KcE47M/3v7fyBg+/0Q+IlgJ7HX3/e7eC/w3sCrONUmMuPuLQPMxm1cBD0VvP0TkP0jCOc6+B4K717n7pujtDmAnMIvgfPbH2/8xSeQgmAVUD7tfwyn8A01hDqw1s41mdnu8i4mTUnevg8h/GKAkzvVMtP9jZluiTUcJ2TQynJnNBc4HXiOAn/0x+w9j+PwTOQhshG2J2Q42skvd/QLgg8Ad0eYDCY7vAQuA5UAd8M24VhNjZpYD/AK4093b413PRBth/8f0+SdyENQAs4fdLwNq41TLhHP32uh1A/AkkaayoKmPtqEOtaU2xLmeCePu9e4+4O6DwA9I4M/fzFKJfAk+4u5PRDcH5rMfaf/H+vknchC8AZxpZvPMLA24CXg6zjVNCDPLjp44wsyygauBbSd+VkJ6Grg1evtW4Kk41jKhhr4Eoz5Kgn7+ZmbAg8BOd//WsD8F4rM/3v6P9fNP2F5DANEuU98BkoEfuvt98a1oYpjZfCJHAQApwKOJvu9m9jPg/USm4K0H7gV+CTwGlANVwCfcPeFOqh5n399PpFnAgUrgC0Nt5onEzN4LvARsBQajm/+OSDt5ED774+3/pxjD55/QQSAiIieXyE1DIiIyCgoCEZGAUxCIiAScgkBEJOAUBCIiAacgEAHMbGDYTI1vjedstWY2d/jMoCKTTUq8CxCZJMLuvjzeRYjEg44IRE4guq7DN8zs9ehlYXT7HDNbH53Ua72ZlUe3l5rZk2a2OXq5JPpSyWb2g+ic8WvNLDNuOyVyDAWBSETmMU1Dnxz2t3Z3Xwn8PyIj1YneftjdzwMeAb4b3f5d4AV3XwZcAGyPbj8T+E93Xwq0Ah+L6d6IjIFGFosAZtbp7jkjbK8EPuDu+6OTex1292lmdoTIgiB90e117j7dzBqBMnfvGfYac4F17n5m9P5dQKq7//ME7JrISemIQOTk/Di3j/eYkfQMuz2Azs/JJKIgEDm5Tw67fiV6+2UiM9oC3Az8Lnp7PfAXEFku1czyJqpIkVOlXyUiEZlm9taw+8+6+1AX0nQze43ID6dPRbd9EfihmX0FaAQ+G93+V8BqM7uNyC//vyCyMIjIpKVzBCInED1HUOHuR+Jdi0isqGlIRCTgdEQgIhJwOiIQEQk4BYGISMApCEREAk5BICIScAoCEZGA+/8y/9Q+eZY3yQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(loss_hist[\"train accuracy\"])\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY8AAAEGCAYAAACdJRn3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAApbklEQVR4nO3deXxU9b3/8dcnKyQQliRgSICECiqyBEHE3WpdqyLd1FqXaov16rW2vb2t7f3d2trF9lZ7a6tWrVZtXVulUBUVva3WikuQsKOyExIhEJZASMjy+f0xJzpiEjIhk5Nk3s/HYx5z5jvnzHxOp/LO+Z7v+R5zd0RERGKRFHYBIiLS8yg8REQkZgoPERGJmcJDRERipvAQEZGYpYRdQLzk5OR4YWFh2GWIiPQoCxYs2OruuQdar9eGR2FhISUlJWGXISLSo5jZ+vasp24rERGJmcJDRERipvAQEZGYKTxERCRmCg8REYmZwkNERGKm8BARkZgpPPbz4GvrmLOoPOwyRES6NYXHfp4o2chTb5eFXYaISLem8NhPUU4ma7fuCbsMEZFuTeGxn6KcTDZW1bCvoSnsUkREui2Fx36KcjJpcthQVRN2KSIi3ZbCYz9FOZkArFPXlYhIqxQe+2kOD533EBFpncJjPwMz0hiUkcoahYeISKsUHi0oyslUt5WISBsUHi0oyumnbisRkTYoPFpQlJPB+7tqqdnXEHYpIiLdksKjBUU5/QBYt1XDdUVEWqLwaIFGXImItE3h0YLCnAwA1m7dHXIlIiLdk8KjBRlpKRyS1UfDdUVEWqHwaIWG64qItC5u4WFmw83s72a2wsyWmdnXg/bBZjbPzN4LngdFbXOjma0ys3fM7Myo9slmtiR473Yzs3jV3axQs+uKiLQqnkceDcC33P0IYBpwrZmNBb4LvOTuo4GXgtcE710EHAmcBdxpZsnBZ90FzARGB4+z4lg3AKNyMtleU8+Omn3x/ioRkR4nbuHh7hXu/nawXA2sAPKB6cCDwWoPAhcEy9OBx9y9zt3XAquAqWaWB2S5+3x3d+ChqG3iRiOuRERa1yXnPMysEJgEvAEMdfcKiAQMMCRYLR/YGLVZWdCWHyzv397S98w0sxIzK6msrDyomotyFR4iIq2Je3iYWT/gSeAGd9/V1qottHkb7R9vdL/H3ae4+5Tc3NzYi40yfFAGSabwEBFpSVzDw8xSiQTHw+7+VNC8OeiKInjeErSXAcOjNi8AyoP2ghba4yotJYnhgzMUHiIiLYjnaCsD7gNWuPttUW/NAS4Pli8HZke1X2Rm6WZWROTE+JtB11a1mU0LPvOyqG3iSvczFxFpWUocP/t44FJgiZmVBm3fA24BnjCzq4ANwOcB3H2ZmT0BLCcyUutad28MtrsGeADoC8wNHnFXmJ3Jm2urcHe6YHSwiEiPEbfwcPdXafl8BcBprWzzE+AnLbSXAOM6r7r2GZWbSc2+Riqr6xiS1aerv15EpNvSFeZtaB6uq2lKREQ+SuHRhsJsDdcVEWmJwqMNwwb2JS0lSeEhIrIfhUcbkpOMwmwN1xUR2Z/C4wA0XFdE5OMUHgdQmJPJhm01NDa1eFG7iEhCUngcwKicTPY1NlG+Y2/YpYiIdBsKjwMoyukHaLiuiEg0hccBfHA/80rdz1xEpJnC4wBy+6XTLz2Fddtqwi5FRKTbUHgcgJlRlJOpbisRkSgKj3aI3M9c3VYiIs0UHu1QlJPJpu17qWtoPPDKIiIJQOHRDqNyMmly2Fil8x4iIqDwaJcPZtet1HkPERFQeLRLYY5m1xURiabwaIcBfVPJzkxj3TaFh4gIKDzarSgnU91WIiIBhUc7FWp2XRGRDyg82qkoJ5Mt1XXsqWsIuxQRkdDFLTzM7H4z22JmS6PaHjez0uCxzsxKg/ZCM9sb9d7voraZbGZLzGyVmd1uZhavmtsySifNRUQ+EM8jjweAs6Ib3P1Cdy9292LgSeCpqLdXN7/n7l+Lar8LmAmMDh4f+cyuohFXIiIfilt4uPsrQFVL7wVHD18AHm3rM8wsD8hy9/nu7sBDwAWdXGq7FGZHwmOdwkNEJLRzHicCm939vai2IjNbaGYvm9mJQVs+UBa1TlnQ1iIzm2lmJWZWUllZ2akF901LZtiAPjryEBEhvPC4mI8edVQAI9x9EvBN4BEzywJaOr/R6v1g3f0ed5/i7lNyc3M7tWCAolzNrisiAiGEh5mlAJ8BHm9uc/c6d98WLC8AVgNjiBxpFERtXgCUd121H1WYreG6IiIQzpHHp4CV7v5Bd5SZ5ZpZcrA8isiJ8TXuXgFUm9m04DzJZcDsEGoGIsN1d+6tZ/uefWGVICLSLcRzqO6jwHzgMDMrM7Orgrcu4uMnyk8CFpvZIuAvwNfcvflk+zXA74FVRI5I5sar5gMZlRtMkKijDxFJcCnx+mB3v7iV9itaaHuSyNDdltYvAcZ1anEd1Dziau3WPUweOSjkakREwqMrzGMwfHAGyUmm4boikvAUHjFITU5ixOAMnTQXkYSn8IhRYXaGznmISMJTeMSoKKcf67buIXLBu4hIYlJ4xKgoN5O99Y1s3lUXdikiIqFReMSoKLt5uO7ukCsREQmPwiNGRbmaXVdEROERo7ysPqSnJGm4rogkNIVHjJKSjCLdklZEEpzCowMKszW7rogkNoVHBxTlZrKxqoaGxqawSxERCYXCowOKcjKpb3Q27dgbdikiIqFQeHRAUY5m1xWRxKbw6IDm8NCIKxFJVAqPDsjOTKN/nxSNuBKRhKXw6AAzDdcVkcSm8OgghYeIJDKFRwcV5WSyacdeausbwy5FRKTLKTw6qCgnE3fYUFUTdikiIl0ubuFhZveb2RYzWxrVdpOZbTKz0uBxTtR7N5rZKjN7x8zOjGqfbGZLgvduNzOLV82x+GC4bqW6rkQk8cTzyOMB4KwW2n/l7sXB41kAMxsLXAQcGWxzp5klB+vfBcwERgePlj6zyxU2D9fdpvAQkcQTt/Bw91eAqnauPh14zN3r3H0tsAqYamZ5QJa7z/fIrfseAi6IS8ExyuqTSk6/dNbqyENEElAY5zyuM7PFQbfWoKAtH9gYtU5Z0JYfLO/f3i0U5WRoxJWIJKSuDo+7gE8AxUAFcGvQ3tJ5DG+jvUVmNtPMSsyspLKy8iBLPbCinEzWqttKRBJQl4aHu29290Z3bwLuBaYGb5UBw6NWLQDKg/aCFtpb+/x73H2Ku0/Jzc3t3OJbUJTTj8rqOqpr6+P+XSIi3UmXhkdwDqPZDKB5JNYc4CIzSzezIiInxt909wqg2symBaOsLgNmd2XNbSnKyQBg3VYN1xWRxJISrw82s0eBU4AcMysDfgCcYmbFRLqe1gFXA7j7MjN7AlgONADXunvz1XfXEBm51ReYGzy6haKcfgCs3baH8QUDQq5GRKTrxC083P3iFprva2P9nwA/aaG9BBjXiaV1mpHZGZihEVciknB0hflB6JOazLABfVm7dXfYpYiIdCmFx0HSBIkikogUHgepOTwi1zCKiCQGhcdBKsrJZFdtA1V79oVdiohIl1F4HKTmCRLVdSUiiUThcZBGD40M1y3duCPcQkREupDC4yAVDMpgfP4A5ixq9cJ3EZFeR+HRCaYXD2Nx2U7WVGrIrogkBoVHJzh3wjDM0NGHiCQMhUcnOGRAH6YVZTOntFxDdkUkIbQrPMws08ySguUxZna+maXGt7SeZXrxMNZs3cPSTbvCLkVEJO7ae+TxCtDHzPKBl4AvE5msUAJnj8sjNdmYXbop7FJEROKuveFh7l4DfAb4jbvPAMbGr6yeZ0BGKqccNoS/LS6nsUldVyLSu7U7PMzsWOAS4JmgLW4z8vZUFxTns3lXHW+s3RZ2KSIicdXe8LgBuBGYFdx7YxTw97hV1UOddsQQMtOSmb1Qo65EpHdrV3i4+8vufr67/zw4cb7V3a+Pc209Tp/UZM4cdwjPLq2grqHxwBuIiPRQ7R1t9YiZZZlZJpG7/b1jZt+Ob2k90/TifKprG/jHO5VhlyIiEjft7bYa6+67gAuAZ4ERwKXxKqonO/4T2WRnpjGnVF1XItJ7tTc8UoPrOi4AZrt7PZH7kMt+UpKTOHdCHi+u2Ex1bX3Y5YiIxEV7w+NuYB2QCbxiZiMBXQ3XivOL86lraOKFZZvDLkVEJC7ae8L8dnfPd/dzPGI98Mm2tjGz+81si5ktjWr7HzNbaWaLzWyWmQ0M2gvNbK+ZlQaP30VtM9nMlpjZKjO73cysY7vadY4aMZDhg/syW3NdiUgv1d4T5gPM7DYzKwketxI5CmnLA8BZ+7XNA8a5+wTgXSLDf5utdvfi4PG1qPa7gJnA6OCx/2d2O2bG9In5vPpeJZXVdWGXIyLS6drbbXU/UA18IXjsAv7Q1gbu/gpQtV/bC+7eELx8HSho6zPMLA/Icvf5Hplx8CEi5126venFw2hyeGaxjj5EpPdpb3h8wt1/4O5rgscPgVEH+d1XAnOjXheZ2UIze9nMTgza8oGyqHXKgrYWmdnM5qOjyspwh8qOHtqfI/Ky1HUlIr1Se8Njr5md0PzCzI4H9nb0S83s+0AD8HDQVAGMcPdJwDeBR8wsC2jp/Earo7zc/R53n+LuU3JzcztaXqeZXjyMhRt2sGFbTdiliIh0qvaGx9eAO8xsnZmtA34LXN2RLzSzy4FzgUuCrijcvc7dtwXLC4DVwBgiRxrRXVsFQI/5U/68icMAmLNIM+2KSO/S3tFWi9x9IjABmBAcIZwa65eZ2VnAd4Dzg1l6m9tzzSw5WB5F5MT4GnevAKrNbFowyuoyYHas3xuW/IF9mVo4mL/qJlEi0svEdCdBd98VXGkOke6lVpnZo8B84DAzKzOzq4gcsfQH5u03JPckYLGZLQL+AnzN3ZtPtl8D/B5YReSIJPo8Sbc3fdIwVm3ZzYqK6rBLERHpNAczrXqb11u4+8UtNN/XyrpPAk+28l4JMC7m6rqJc8bl8YPZy5hduomxw7LCLkdEpFMczD3M1Q/TDoMy0zh5TC5zFpXTpJtEiUgv0WZ4mFm1me1q4VENDOuiGnu884uHUbGzlrfWVR14ZRGRHqDN8HD3/u6e1cKjv7vrToLtdPrYofRNTdY1HyLSaxxMt5W0U0ZaCmccOZRnl1Swr6Ep7HJERA6awqOLTC8exo6aev75nm4SJSI9n8Kji5w4OpdBGanM1k2iRKQXUHh0kdTkJD49IY95yzezp67hwBuIiHRjCo8uNL04n731jcxbrptEiUjPpvDoQpNHDCJ/YF9ml2quKxHp2RQeXSgpyThv4jBeeW8r23brJlEi0nMpPLrY9OJhNDY5zy59P+xSREQ6TOHRxQ4/pD9jhvZjjrquRKQHU3h0MTPjs0cV8Na67Ty5oOzAG4iIdEMKjxBceUIRx30im+8+tZg312q+KxHpeRQeIUhNTuKuSyYzfHAGV/+xhHVb94RdkohITBQeIRmQkcr9lx+NA1c++BY7a+rDLklEpN0UHiEqzMnk7i9NZmNVDdc8vID6Rk2aKCI9g8IjZMeMyuaWz0zgtdXb+O/ZS3WvcxHpEXRPjm7gs5MLWLN1N3f8fTWjcvrx1ZNGhV2SiEibFB7dxLdOP4y1W/fw07krGJmdwRlHHhJ2SSIirYpbt5WZ3W9mW8xsaVTbYDObZ2bvBc+Dot670cxWmdk7ZnZmVPtkM1sSvHe7mVm8ag5TUpJx6+eLmZA/gK8/VsrSTTvDLklEpFXxPOfxAHDWfm3fBV5y99HAS8FrzGwscBFwZLDNnWaWHGxzFzATGB089v/MXqNvWjL3Xj6FQRmpfOXBEt7fWRt2SSIiLYpbeLj7K8D+V8BNBx4Mlh8ELohqf8zd69x9LbAKmGpmeUCWu8/3yJnkh6K26ZWG9O/DfVccTXVtPV956C1q9uneHyLS/XT1aKuh7l4BEDwPCdrzgY1R65UFbfnB8v7tLTKzmWZWYmYllZU993avR+Rl8ZsvTmJ5+S5ueKyUpiaNwBKR7qW7DNVt6TyGt9HeIne/x92nuPuU3NzcTisuDKcePpT/+vRYXli+mZ8/vzLsckREPqKrR1ttNrM8d68IuqS2BO1lwPCo9QqA8qC9oIX2hPDl4wtZs3U3d7+8hlE5mVx49IiwSxIRAbr+yGMOcHmwfDkwO6r9IjNLN7MiIifG3wy6tqrNbFowyuqyqG16PTPjpvOO5MTROXx/1lJeW7017JJERID4DtV9FJgPHGZmZWZ2FXALcLqZvQecHrzG3ZcBTwDLgeeAa929Mfioa4DfEzmJvhqYG6+au6OU5CTuuOQoinIy+dofF7BqS3XYJYmIYL11OowpU6Z4SUlJ2GV0mo1VNcy48zX6piUx69+OJ6dfetgliUgvZGYL3H3KgdbrLifM5QCGD87gvsunUFldx1ceLKG2vvHAG4mIxInCoweZOHwgv75oEovKdvCNxzWEV0TCo/DoYc488hC+f84RzF36Prc8pyG8IhIOTYzYA111QhEbqmq455U1jBicwZemjQy7JBFJMAqPHsjM+O9zx1K2fS//PXsp+YP68snDhhx4QxGRTqJuqx4qJTmJ31w8iSPysrju4bdZXr4r7JJEJIEoPHqwzPQU7r/iaLL6pnLlA29RsXNv2CWJSIJQePRwQ7P6cP8VR7O7roErHyhhd51m4RWR+FN49AJH5GVxxyVH8e7maq575G0aGpvCLklEejmFRy9x8phcbp4+jn+8U8lNf1tGb505QES6B4226kW+eMwI1lft4e6X1zBycCZfPWlU2CWJSC+l8OhlvnPm4ZRV7eWnc1dQMKgvZ4/PC7skEemFFB69TFKScesXJlKxcy83PF5Kxc5aPju5gAF9U8MuTUR6EZ3z6IX6pCZz72VTmFAwgB89vZxjfvoi3/7zIko37tC5EBHpFJqSvZdbumknD7+xgdmlm6jZ18i4/CwuOWYk508cRma6DjxF5KPaOyW7wiNBVNfW89fSch5+fT0r36+mX3oKMyblc8m0ERx+SFbY5YlIN6HwUHi0yN15e8MOHn59PU8vqWBfQxNTRg7ikmkjOHtcHn1Sk8MuUURCpPBQeBzQ9j37ePLtMh5+YwNrt+5hYEYqFx49nGs/eShZfXSCXSQRKTwUHu3m7sxfvY0/vbGe55a+z5D+fbj5gnGcPnZo2KWJSBfTbWil3cyM4w7N4c5LJvPUvx3PwIxUvvpQCdc+8jaV1XVhlyci3VCXh4eZHWZmpVGPXWZ2g5ndZGabotrPidrmRjNbZWbvmNmZXV1zIikePpA5153Af5wxhnnLNvOp217miZKNGuIrIh8RareVmSUDm4BjgC8Du939l/utMxZ4FJgKDANeBMa4e2Nbn61uq4O3astubnxqMW+t287xh2bzsxkTGJGdEXZZIhJHPaXb6jRgtbuvb2Od6cBj7l7n7muBVUSCROLs0CH9eHzmsdx8wTgWbdzJGf/7Mve+skaz9opI6OFxEZGjimbXmdliM7vfzAYFbfnAxqh1yoK2jzGzmWZWYmYllZWV8ak4wSQlGZdOG8m8b57ECYfm8JNnVzDjztdYVr4z7NJEJEShdVuZWRpQDhzp7pvNbCiwFXDgZiDP3a80szuA+e7+p2C7+4Bn3f3Jtj5f3Vadz915ZkkFN81Zxvaaeq4+aRTXnza6xWtD3J1dtQ1UVteyZVcdm5ufd9Wxc289F08dzpTCwSHshYi0pb3dVmHOT3E28La7bwZofgYws3uBp4OXZcDwqO0KiISOdDEz49wJwzjh0Bx+/MwK7vzHauYufZ/PHpXP1t37qKyuY/OuWrZU17Glupba+o93b/VNTSYlyZizaBO/+NwEZkwqCGFPRORghRkeFxPVZWVmee5eEbycASwNlucAj5jZbUROmI8G3uzKQuWjBmak8cvPT+SC4nxunLWYX77wLv3SUxjSP50hWekUDx/IkP7pDM3qw5CsdHKbl/un0y89hZ176/nanxbwjccXsXZrDd/41GjMLOzdEpEYhNJtZWYZRM5jjHL3nUHbH4FiIt1W64Crm8PEzL4PXAk0ADe4+9wDfYe6rbpGY5NTW98Y8ySL+xqa+N6sJfxlQRnTi4fx889O0NQoIt2ArjBXeHR77s6d/1jN/zz/DlNGDuLuSyeT3S897LJEElpPGaorCczMuPaTh/LbL05i8aadzLjzNVZX7g67LBFpB4WHhO7cCcN4bOY09tQ1MOOOf/Ha6q1hl9RuvfXIXeRAFB7SLRw1YhB/vfZ4hmT14bL73uSJko0H3ihE7s7vXl7NpJvnMWeRBv9J4lF4SLcxfHAGT15zHNNGZfOff1nML55bSVNT9/vLvrHJ+cGcZdwydyXJZlz/6EJ+89J7OgqRhKLwkG5lQN9U/vDlo7l46nDu/Mdq/v2xhdTWtzmNWZeqrW/kmj8t4KH567n6pFH867unMmNSPrfOe5dvPbGIuobuU6tIPOkm1tLtpCYn8dMZ4ynKyeRnc1dSvmMv9142hZw2RmI1Njn1jU3BI7I8oG9qpw7/rdqzj688+BYLN+7gpvPGcsXxRQDc9oWJFGZn8qsX36Vs+17uvnQygzLTOu17RbojDdWVbu25pRXc8Hgp6SnJDMxIpaHR2dfYREMQEs3LLfVuZWem8a0zDuPCo4eTnHRwFyFu2FbDFX94k7Ide/n1hcWcPT7vY+vMLt3Et/+8mGED+3D/FUczKrffQX2nSBh0nYfCo9dYXLaD+19diwMpSUmkpRipyUmkJCWRmmKkJSdFXid/uJyUZMwp3cRb67ZzRF4WPzhvLNNGZXf4+6984C3qG537Lp/S5pxcJeuqmPnHBTQ2OXdfOrnD3ykSFoWHwiPhuTtPL67gZ8+uoHxnLeeMP4Qbzz6C4YPbf0+Sv7+zhWsffptBGWk8eOVUDh1y4KOJDdtq+PIDb7KhqoafzhjP56cMP+A2It2FLhKUhGdmnDdxGC996xS+8akx/N/KLZx228v88vl32FPXcMDtH39rA195sISinExmXXtcu4IDYER2Bk/92/FMLRrMt/+ymP95vnuOGpPubVdtPTc/vZwN22rCLqVFOvKQhFG+Yy8/f24ls0vLGZqVznfPPpzpE/NJ2u98iLvzvy++x69feo+TxuRy5yVH0S/GubsA6hub+K9ZS3m8ZCOfnpDHrZ+f2K4T+HvqGlhctpPSjTso3bidFRXVnHr4EG4853DSUzT/VyJwd657dCHPLK7g6MJBPD7z2I/9/zRe1G2l8JBWLFhfxQ//tpzFZTspHj6QH5w3lkkjIvceq29s4vuzlvBESRmfm1zAzz4zntTkjh+guzv3vLKGW55bycSCgdx72RRy+384aqyxyXl3czWlG3ewaOMOSjfu4N3N1R8MACjMzmBEdiavvFvJhIIB3PHFo2LqdpOe6eE31vP9WUs5dlQ289ds48cXjONL00Z2yXcrPBQe0oamJufJt8v4xfPvUFldx2cm5XPdqYfyw78t5+V3K7n+1EP5xuljOm2q+OZRY9mZ6Xzz9DG8u6Wa0g07WLJpJzX7IteGDMxIZWLBQIqHD6R4xECKCwZ+MOT3+WXv8x9/XoQBt32hmE+NHdopdUn3s6JiF9Pv+BfHFA3mgS9P5bL732Dxxp3M++bJHDKgT9y/X+Gh8JB22F3XwB1/X8V9/1zLvsYmkgx+fMF4vnjMiE7/rsVlO7jqwRIqq+tITTbG5mV9GBTDB1GYndFmWG3YVsM1Dy9gWfkurj55FN8+4zBSDuKoSLqfPXUNnP/bV9lV28Dcr59ITr901m/bwxm/eoWTx+Ryz2UH/Df9oCk8FB4Sg/Xb9vC7l9dwxpFD+eRhQ+L2PTtq9rGhqoYxQ/t36ALG2vpGfvT0ch55YwNTCwfzmy9OYmhW/P8ala7xrScW8dTCMh6+6hiOOzTng/bfvbyaW+au5HdfOoqzxn38GqPOpPBQeEgv9teFm7jxqSVkpifz64smcXzUPzTSMz25oIxv/XkR1582mm+ePuYj7zU0NjH9jn+xpbqOF795MgP6psatDg3VFenFLpiUz5zrjmdgRhpfuu8Nbn/pPQ0H7sFWbdnN/5u9lGOKBvP100Z/7P2U5CR+/tkJVO3Zxy1zV4ZQ4ccpPER6qNFD+zP72uOZPnEYt817l8v/8CbbdteFXZbEqLa+keseeZs+qZGjyNam0hmXP4CrTiji0Tc38PqabV1c5ccpPER6sMz0FH51YTE/nTGeN9ZW8enbX2XB+qqwy5IY3Pz0cla+X82tX5h4wNFU3/jUGEYMzuB7Ty0JfbbpUMLDzNaZ2RIzKzWzkqBtsJnNM7P3gudBUevfaGarzOwdMzszjJpFuisz44vHjOCpa44jLSWJC+9+nXteWR3X6eGra+t5ZnEF722ujtt3JIJnFlfw8BsbuPqkUe0aqNE3LZmfzBjHmq17+O3/reqCClsXyglzM1sHTHH3rVFtvwCq3P0WM/suMMjdv2NmY4FHganAMOBFYIy7t/lfhk6YSyLaubeeb/95ES8s30xWnxTOHpfH9OJhHDMq+6BnFm5obOKfq7Yy6+1NvLD8fWrrm0hJMq4+eRT/furoTp3+PhFs2FbDp2//J4cO7ccTVx8b08Wo33piEbNLN/H09Sdw+CFZnVpXtx5t1Up4vAOc4u4VZpYH/MPdDzOzGwHc/WfBes8DN7n7/La+Q+EhicrdefndSuaUlvP8svfZs6+RIf3TOW/iMKYXD2N8/oB2X/zo7iwr38WshZuYXVrO1t11DOibynkT8zhnXB5PLdzEXxaUUZSTyU9njOfYT2gW4fbY19DE5373Guu27uGZ60+MedaA7Xv28anbXqZgcAZPXXPcQf9hEK294RHWzaAceMHMHLjb3e8Bhrp7BUAQIM3HcPnA61HblgVtItICM+OUw4ZwymFD2LuvkZdWbmZ2aTl/nL+e+15dS1FOJudPHMb5xcP4RCv3HKnYuZe/Lixn1sIy3t28m9Rk49TDhzBjUgGfPDz3gzm2jjs0hxmT8vnerCVcfO/rXDhlON875wgGZBzcUNKGxibmLn2f37+6lk3ba/jMUQVccswIRmZnHtTndhc/f24li8t28rsvdWy6mUGZafz3eWP5+mOlPPjaOq48oSgOVbYtrCOPYe5eHgTEPODfgTnuPjBqne3uPsjM7gDmu/ufgvb7gGfd/ckWPncmMBNgxIgRk9evX98FeyPSM+ysqee5ZRXMLi1n/pptuMO4/CymT8zn3Il59O+TynNL32fWwjJeWx15/6gRA5lxVAHnjs9r8+6Ie/c18uuX3uPef65hUEYaN50/lk+Pz4t5epddtfU8/uZGHnhtHZt27KUoJ5PRQ/rx0sotNLlz8phcLp02klMOG9Kpf213pReXb+YrD5Vw+bEj+eH0cR3+HHfnyw+8xZtrq3jhGydRMKhz5jzr1t1WHynA7CZgN/BV1G0l0iU276rlb4vK+duichaV7cQM0pKTqGtoYvjgvsyYVMCMSfkU5cT2l/6y8p3c+NQSFpft5LTDh3DzBeMYNrDvAbfbWFXDH/61jsff2sCefY1MGzWYr5wwilMPH0JSkrF5Vy2PvrmBR97YwJbqOgoG9eWSY0Zy4dHDGdyDbvlbvmMv59z+T/IH9uXJa4476PNEZdtrOONXrzC1aDB/uOLoTpmLrduGh5llAknuXh0szwN+BJwGbIs6YT7Y3f/TzI4EHuHDE+YvAaN1wlykc6zduoc5peVsr9nHuRPymDxy0EH9I9TQ2MQDr63j1hfeJcng22cexqXHFrZ4pLBg/Xbue3UNzy19n6Tg/itXnVDEuPwBLX52fWMT85Zv5qH563h9TRVpKUmcOz6PLx07kknDB3baRJbx0NDYxEX3vM6Kil08ff2JMQdza+5/dS0/eno5v76omOnFB9+j353DYxQwK3iZAjzi7j8xs2zgCWAEsAH4vLtXBdt8H7gSaABucPe5B/oehYdIuDZW1fBff13Ky+9WUjx8ILd8djyHH5JFQ2MTzy/bzO9fXcPCDTvI6pPCJdNGcvmxhTHNGvve5mr++Pp6nnp7E7vrGhiXn8Wl00Zy/sR8+qbFZ+RXU5OzZ18D1bUN1OxroLa+idr6RuoaPvpcW99EXcNHn1dtqebFFVs67R/5Zo1Nzmfueo2yqhpe/ObJbXYvtke3DY+uovAQCZ+7M2dROT/823J27a1nxqR8Xlu9jU079jIyO4Mrjy/ic5MLyOzAzbaa7a5r4K8LN/HH+et5Z3M1WX1SGJc/gLSUpMg97VOSSA/ubZ+WEnk0L6enJJGabKQkJVEThEJ1XfBcW8/u2g+Xq+sa2F3XQEf+yWz+ri8eM4Ibzz6iw/vampXv7+Lc219lenE+t35h4kF9lsJD4SHSbWzfs48fP7OCJ98uY2rhYK46sYhPHTG0U096uztvrdvOo29uYGNVDfsam9jX0PTBc/0Hz/5B+/7SU5Lo3yeF/n1S6ZeeEiyn0C89lf59Usjqk0K/4P3M9BT6pCTRJzWZ9OD5o8tJpKdEXnfFXQB/+fw7/Pbvq/jjVVM5cXRuhz9H4aHwEOl2ausbu83FhO4eCZLGJhoam+ibltyjb/NbW9/IOb/+J/VNTbxww8kd7rrTrLoi0u10l+CAyPUwaSlJ9EtPYWBGWo8ODoj8b/uzz4xnQv7ALpn3KqyLBEVEpJMdMyqbY0Z1zVX+OvIQEZGYKTxERCRmCg8REYmZwkNERGKm8BARkZgpPEREJGYKDxERiZnCQ0REYtZrpycxs0qgo3eDygG2HnCt3imR9x0Se/8Ted8hsfc/et9HuvsBJ8fqteFxMMyspD1zu/RGibzvkNj7n8j7Dom9/x3Zd3VbiYhIzBQeIiISM4VHy+4Ju4AQJfK+Q2LvfyLvOyT2/se87zrnISIiMdORh4iIxEzhISIiMVN4RDGzs8zsHTNbZWbfDbuermZm68xsiZmVmlmvvoevmd1vZlvMbGlU22Azm2dm7wXPg8KsMZ5a2f+bzGxT8PuXmtk5YdYYL2Y23Mz+bmYrzGyZmX09aO/1v38b+x7zb69zHgEzSwbeBU4HyoC3gIvdfXmohXUhM1sHTHH3Xn+hlJmdBOwGHnL3cUHbL4Aqd78l+ONhkLt/J8w646WV/b8J2O3uvwyztngzszwgz93fNrP+wALgAuAKevnv38a+f4EYf3sdeXxoKrDK3de4+z7gMWB6yDVJnLj7K0DVfs3TgQeD5QeJ/EfVK7Wy/wnB3Svc/e1guRpYAeSTAL9/G/seM4XHh/KBjVGvy+jg/6g9mAMvmNkCM5sZdjEhGOruFRD5jwwYEnI9YbjOzBYH3Vq9rttmf2ZWCEwC3iDBfv/99h1i/O0VHh+yFtoSrU/veHc/CjgbuDbo2pDEcRfwCaAYqABuDbWaODOzfsCTwA3uvivserpSC/se82+v8PhQGTA86nUBUB5SLaFw9/LgeQswi0hXXiLZHPQJN/cNbwm5ni7l7pvdvdHdm4B76cW/v5mlEvnH82F3fypoTojfv6V978hvr/D40FvAaDMrMrM04CJgTsg1dRkzywxOoGFmmcAZwNK2t+p15gCXB8uXA7NDrKXLNf/DGZhBL/39zcyA+4AV7n5b1Fu9/vdvbd878ttrtFWUYHja/wLJwP3u/pNwK+o6ZjaKyNEGQArwSG/efzN7FDiFyFTUm4EfAH8FngBGABuAz7t7rzyp3Mr+n0Kk28KBdcDVzecAehMzOwH4J7AEaAqav0ek779X//5t7PvFxPjbKzxERCRm6rYSEZGYKTxERCRmCg8REYmZwkNERGKm8BARkZgpPEQ6yMwao2YhLe3MmZjNrDB6xluR7iYl7AJEerC97l4cdhEiYdCRh0gnC+6L8nMzezN4HBq0jzSzl4LJ514ysxFB+1Azm2Vmi4LHccFHJZvZvcF9F14ws76h7ZTIfhQeIh3Xd79uqwuj3tvl7lOB3xKZtYBg+SF3nwA8DNwetN8OvOzuE4GjgGVB+2jgDnc/EtgBfDaueyMSA11hLtJBZrbb3fu10L4OONXd1wST0L3v7tlmtpXIjXjqg/YKd88xs0qgwN3roj6jEJjn7qOD198BUt39x12wayIHpCMPkfjwVpZbW6cldVHLjegcpXQjCg+R+Lgw6nl+sPwakdmaAS4BXg2WXwKugcjtkM0sq6uKFOko/SUj0nF9zaw06vVz7t48XDfdzN4g8gfaxUHb9cD9ZvZtoBL4ctD+deAeM7uKyBHGNURuyCPSbemch0gnC855THH3rWHXIhIv6rYSEZGY6chDRERipiMPERGJmcJDRERipvAQEZGYKTxERCRmCg8REYnZ/wdwAaq2VO9I5AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(loss_hist[\"train loss\"])\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy%:  84.9 == 8490 / 10000\n"
     ]
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    model.eval()\n",
    "    \n",
    "    y_true_test = []\n",
    "    y_pred_test = []\n",
    "    \n",
    "    for batch_idx, (img, labels) in enumerate(testloader):\n",
    "        img = img.to(device)\n",
    "        label = label.to(device)\n",
    "    \n",
    "        preds = model(img)\n",
    "        \n",
    "        y_pred_test.extend(preds.detach().argmax(dim=-1).tolist())\n",
    "        y_true_test.extend(labels.detach().tolist())\n",
    "        \n",
    "    total_correct = len([True for x, y in zip(y_pred_test, y_true_test) if x==y])\n",
    "    total = len(y_pred_test)\n",
    "    accuracy = total_correct * 100 / total\n",
    "    \n",
    "    print(\"Test Accuracy%: \", accuracy, \"==\", total_correct, \"/\", total)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving Model Weights "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model, \"../../trained_models/vgg16_cifar10.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
